VimUnDo 7JSZj4LZ	   O       E         <      <  <  <    ^-   S _                                                                                                                                                                                                                                                                                                                                                                                          ^                        5_                                                                                                                                                                                                                                                                                                                                                                                        ^                     
PPSN 20205_                                                                                                                                                                                                                                                                                                                                                                                         ^                                                        5_                       )                                                                                                                                                                                                                                                                                                                                                                 ^                      ,Automatic Configuration of a Multi-Objective   *Local Search for Imbalanced Classification5_                       ,                                                                                                                                                                                                                                                                                                                                                                 ^                                     5_                                                                                                                                                                                                                                                                                                                                                                                         ^                                       5_                    
   &                                                                                                                                                                                                                                                                                                                                                                 ^         	              '- p. 1, introduction, 1st paragraph: ""5_      	              
   .                                                                                                                                                                                                                                                                                                                                                                 ^         	              .- p. 1, introduction, 1st paragraph: "aims to"5_      
           	   
   -                                                                                                                                                                                                                                                                                                                                                                 ^         	              0- p. 1, introduction, 1st paragraph: "aims to" -5_   	              
   
   8                                                                                                                                                                                                                                                                                                                                                                 ^        	              8- p. 1, introduction, 1st paragraph: "aims to predict" -5_   
                                                                                                                                                                                                                                                                                                                                                                                      ^                                      5_      
                 )                                                                                                                                                                                                                                                                                                                                                                 ^                      +Why does MOCA-I returns a single ruleset ??5_                 
      *                                                                                                                                                                                                                                                                                                                                                                 ^                     *Why does MOCA-I returns a single ruleset??5_   
                                                                                                                                                                                                                                                                                                                                                                                      ^-                                  5_                       %                                                                                                                                                                                                                                                                                                                                                                 ^?                    3First, stated that practitonners want pareto-front 5_                       F                                                                                                                                                                                                                                                                                                                                                                 ^F                      Then,    fWhy does MOCA-I returns a single ruleset?? Why not using mono-objective with this harmonic mean, then?               FFirst, stated that practitonners need to navigate within pareto-front 5_                       1                                                                                                                                                                                                                                                                                                                                                                 ^R                     lThen, why does MOCA-I returns a single ruleset?? Why not using mono-objective with this harmonic mean, then?5_                       :                                                                                                                                                                                                                                                                                                                                                                 ^V                                     5_                                                                                                                                                                                                                                                                                                                                                                                         ^#                                      5_                                                                                                                                                                                                                                                                                                                                                                                        ^1                      ,Section 3. I would mention that the optimal 5_                       ?                                                                                                                                                                                                                                                                                                                                                                 ^9                      GSection 3. I would mention that, because of stochasticity, the optimal 5_                       N                                                                                                                                                                                                                                                                                                                                                                 ^<                      NSection 3. I would mention that, because of stochasticity, the chosen optimal 5_                                                                                                                                                                                                                                                                                                                                                                                         ^K                    gSection 3. I would mention that, because of stochasticity, the chosen optimal configuration may not be 5_                                                                                                                                                                                                                                                                                                                                                                                         ^                      5_                       4                                                                                                                                                                                                                                                                                                                                                                 ^                      5or a bi-objective problem, as suggested in section 3.5_                       6                                                                                                                                                                                                                                                                                                                                                                 ^   	                  8or a bi-objective problem, as suggested in section 3 ().5_                       _                                                                                                                                                                                                                                                                                                                                                                 ^l   
                                  5_                                                                                                                                                                                                                                                                                                                                                                                         ^oJ       
             5_                                                                                                                                                                                                                                                                                                                                                                                         ^o                           
       5_                                                                                                                                                                                                                                                                                                                                                                                        ^o           
         - "the percentage   )of the class to predict for each dataset"5_                                                                                                                                                                                                                                                                                                                                                                                         ^o           
         ;- "the percentage of the class to predict for each dataset"5_      !                                                                                                                                                                                                                                                                                                                                                                                   ^o        
            4- p. 7, sec. 4, "the method in [6]" -> add a period.5_       "           !      J                                                                                                                                                                                                                                                                                                                                                                 ^o           
         J- p.7, sec. 4.1: "the percentage of the class to predict for each dataset"5_   !   #           "      O                                                                                                                                                                                                                                                                                                                                                                 ^o           
         P- p.7, sec. 4.1: "the percentage of the class to predict for each dataset" -> ""5_   "   $           #      `                                                                                                                                                                                                                                                                                                                                                                 ^o           
         k- p.7, sec. 4.1: "the percentage of the class to predict for each dataset" -> "the percentage of the class"5_   #   %           $      J                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^o           
         x- p.7, sec. 4.1: "the percentage of the class to predict for each dataset" -> "the percentage of occurence of the class"5_   $   &           %                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^p           
         J- p.7, sec. 4.1: "the percentage of the class to predict for each dataset"5_   %   '           &      S                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p          
         S- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset"5_   &   (           '      Z                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p   
       
         a- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not clear.5_   '   )           (      [                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p          
         l- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediatly clear.5_   (   *           )      l                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p-                    m- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear.5_   )   +           *                                                                                                                                                                                                                                                                                                                                         J       
          v   w    ^p5           
         s- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear and si     doubled in the next sentence.5_   *   ,           +      q                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p6           
         - p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear and si doubled in the next sentence.5_   +   -           ,      r                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p7          
         - p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear and i doubled in the next sentence.5_   ,   .           -      t                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^p:                    - p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear and is doubled in the next sentence.5_   -   /           .      	                                                                                                                                                                                                                                                                                                                                   J       
          v   w    ^p?          
         s- p.7, sec. 4.1: sentence "the percentage of the class to predict for each dataset" is not immediately clear and is      repeated in the next sentence.5_   .   0           /   
                                                                                                                                                                                                                                                                                                                                       J          w       v   w    ^rR                     5_   /   1           0   
                                                                                                                                                                                                                                                                                                                                      J          w       v   w    ^rb                    - p.8, sec. 4.2: ""5_   0   2           1   
   %                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^rf                   %- p.8, sec. 4.2: "in order to fix it"5_   1   3           2                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^r                       5_   2   4           3                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^r                      Sec. 4.2 does the running 5_   3   5           4                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^r                      Sec. 4.2 does the running 5_   4   6           5                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^r                      Sec. 4.2 does the running time5_   5   7           6      "                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^r                     "Sec. 4.2 does the set running time5_   6   8           7      J                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^r                   5_   7   9           8                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^s                      5_   8   :           9      P                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^s5                   5_   9   ;           :   
                                                                                                                                                                                                                                                                                                                                      J          w       v   w    ^sP       
                
          5_   :   <           ;      P                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^u                                  5_   ;   =           <                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^u                    The runs5_   <   >           =                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^u                    !If wallclock times, have the runs5_   =   ?           >      9                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^u                    9If wallclock times, have the preliminary experiments runs5_   >   @           ?                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^u                    uIf wallclock times, have the preliminary experiments be conducted in the same setting than the experiments (i.e. full   load on 24 cores).5_   ?   A           @                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^u                    If wallclock times, have the preliminary experiments be conducted in the same setting than the experiments (i.e. full load on 24 cores).5_   @   B           A                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^u                    load on 24 cores) ?5_   A   C           B                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^u                   uIf wallclock times, have the preliminary experiments be conducted in the same setting than the experiments (i.e. full   load on 24 cores)?5_   B   D           C                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^v,                                      5_   C   E           D                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^v2                      p.9 results 5_   D   F           E                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^v6                     p.9 indicate why results 5_   E   G           F                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vB                      ?p.9 indicate why results does not show evidence of overfitting.             5_   F   H           G                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vB                    ?p.9 indicate why results does not show evidence of overfitting.             5_   G   I           H                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   H   J           I                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   I   K           J                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   J   L           K                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   K   M           L                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   L   N           M                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   M   O           N                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   N   P           O                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   O   Q           P                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   P   R           Q                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vC                    ?p.9 indicate why results does not show evidence of overfitting.             5_   Q   S           R                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vD                    ?p.9 indicate why results does not show evidence of overfitting.             5_   R   T           S                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^vE                   ?p.9 indicate why results does not show evidence of overfitting.5_   S   U           T      @                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^vL                   A- p.9 indicate why results does not show evidence of overfitting.5_   T   V           U                                                                                                                                                                                                                                                                                                                                          J          w       v   w    ^xv                                                     5_   U   W           V      ,                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^x                     JParameters of the Friedman and Wilcoxon testshould be indicated somewhere.5_   V   X           W      5                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^x                                     5_   W   Y           X                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^y                     wTable 5 does not show "the best configurations […] in bold", given that there are some lines where all configurations   are in bold.5_   X   Z           Y                                                                                                                                                                                                                                                                                                                                         J          w       v   w    ^y2                      Table 5 does not show "the best configurations […] in bold", given that there are some lines where all configurations are in bold.5_   Y   [           Z       O                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^yG                      Xare in bold. Please take care of describing and interpreting statistical tests carefully5_   Z   \           [       ]                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^yL                     ]are in bold. Please take care of describing and interpreting statistical tests very carefully5_   [   ]           \       /                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y`                                        5_   \   ^           ]   "   V                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y        !              ]While it is sufficiently clear that AC allow for improvement in terms of Pareto front metrics5_   ]   _           ^   "   e                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y        !              eWhile it is sufficiently clear that AC allow for improvement in terms of Pareto front quality metrics5_   ^   `           _   #   -                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y        "              3it is a lot less clear that there is an improement 5_   _   a           `   #   (                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y        "              4it is a lot less clear that there is an improvement 5_   `   b           a   #   4                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y       "              4it is a lot less clear that there is an improvement 5_   a   c           b   #   N                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y        "              Nit is a lot less clear that there is an improvement on the operational result.5_   b   d           c   $   *                                                                                                                                                                                                                                                                                                                                   J          w       v   w    ^y   !    $                  $            5_   c   e           d   %                                                                                                                                                                                                                                                                                                                                   %          %   
       v   
    ^y        $              8A synthetic view on results shown on table 5 would help.5_   d   f           e   %                                                                                                                                                                                                                                                                                                                                   %          %   
       v   
    ^z        $              0A s view on results shown on table 5 would help.5_   e   g           f   %                                                                                                                                                                                                                                                                                                                                   %          %   
       v   
    ^z
        $              6A consise view on results shown on table 5 would help.5_   f   h           g   %                                                                                                                                                                                                                                                                                                                                   %          %   
       v   
    ^z   #    $              6A concise view on results shown on table 5 would help.5_   g   i           h   %   9                                                                                                                                                                                                                                                                                                                                %          %   
       v   
    ^z   $    %            5_   h   j           i      	                                                                                                                                                                                                                                                                                                                                %          %   
       v   
    ^z,   %          '                &    5_   i   k           j      ,                                                                                                                                                                                                                                                                                                                                &          &   
       v   
    ^z:              '      ,- Figure 3 legend is a little bit too small.5_   j   l           k      H                                                                                                                                                                                                                                                                                                                                &          &   
       v   
    ^zI              '      Q- Figure 3 legend is a little bit too small. The legend should indicate the axis.5_   k   m           l      ^                                                                                                                                                                                                                                                                                                                                &          &   
       v   
    ^zO              '      _- Figure 3 legend is a little bit too small. The legend should indicate how to read the figure.5_   l   n           m      `                                                                                                                                                                                                                                                                                                                                &          &   
       v   
    ^zP   &          '      b- Figure 3 legend is a little bit too small. The legend should indicate how to read the figure ().5_   m   o           n      ?                                                                                                                                                                                                                                                                                                                                &          &   
       v   
    ^zi   '          '      n- Figure 3 legend is a little bit too small. The legend should indicate how to read the figure (axis, labels).5_   n   p           o                                                                                                                                                                                                                                                                                                                                       &          &   
       v   
    ^              '       5_   o   q           p                                                                                                                                                                                                                                                                                                                                      &          &   
       v   
    ^J              '      overall: 1 weak accept5_   p   r           q                                                                                                                                                                                                                                                                                                                                       ,          ,   
       v   
    ^R              -       5_   q   s           r      [                                                                                                                                                                                                                                                                                                                                -          -   
       v   
    ^|              .      ^This paper describes an algorithm configuration method which targets multi-objective algorihm.5_   r   t           s      E                                                                                                                                                                                                                                                                                                                                -          -   
       v   
    ^~              .      _This paper describes an algorithm configuration method which targets multi-objective algorithm.5_   s   u           t      a                                                                                                                                                                                                                                                                                                                                -          -   
       v   
    ^           	   .      aThis paper describes an algorithm configuration method which targets a multi-objective algorithm.5_   t   v           u                                                                                                                                                                                                                                                                                                                                      .          .   
       v   
    ^Ʉ           	   /      
The algorithm5_   u   w           v                                                                                                                                                                                                                                                                                                                                      .          .   
       v   
    ^Ʌ           	   /      The selected algorithm5_   v   x           w      l                                                                                                                                                                                                                                                                                                                                .          .   
       v   
    ^ɱ           	   /      mThe selected algorithm solves a classification problem which is better addressed with rule-based classifiers.5_   w   y           x      \                                                                                                                                                                                                                                                                                                                                .          .   
       v   
    ^ɷ           
   /      \The selected algorithm solves a classification problem which is better addressed with rules.5_   x   z           y   	                                                                                                                                                                                                                                                                                                                                   /          /   
       v   
    ^           
   0      !The work focus on the performance5_   y   {           z   	   5                                                                                                                                                                                                                                                                                                                                /          /   
       v   
    ^           
   0      5The work focus on the algorithm selection performance5_   z   |           {   	   	                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^           
   0      ;The work focus on the algorithm selection performances and 5_   {   }           |   	   F                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^           
   0      FThe work presents results on the algorithm selection performances and 5_   |   ~           }   	                                                                                                                                                                                                                                                                                                                                    	   	       	   
       v   
    ^           
   0      rThe work presents results on the algorithm selection performances and on the terminal classification performances.5_   }              ~   	   J                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^   (          0      zThe work presents results on an indirect algorithm selection performances and on the terminal classification performances.5_   ~                                                                                                                                                                                                                                                                                                                                                     	   	       	   
       v   
    ^              2             
   1    5_                                                                                                                                                                                                                                                                                                                                                        	   	       	   
       v   
    ^           
   3       5_                                                                                                                                                                                                                                                                                                                                                       	   	       	   
       v   
    ^)           
   3      The suject of the work is 5_                                                                                                                                                                                                                                                                                                                                                       	   	       	   
       v   
    ^*           
   3      The subject of the work is 5_                       A                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^@           
   3      ZThe subject of the work is in scope and the works brings new data to the state of the art.5_                       ^                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^D              3      ^The subject of the work is in scope and the works brings new insights to the state of the art.5_                    
                                                                                                                                                                                                                                                                                                                                    	   	       	   
       v   
    ^"              4       5_                    
   +                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^C              4      ?The operational problem is interesting and rule-based problems 5_                    
                                                                                                                                                                                                                                                                                                                                   	   	       	   
       v   
    ^H              4      %The operational, rule-based problems 5_                    
   &                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^I   )          4      &The operational, rule-based, problems 5_                    
   @                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^˟              5      AThe operational, rule-based, problems is an interesting use-case.5_                    
   +                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^˱              5      `The operational, rule-based, problems is an interesting use-case for multi-objective approaches.5_                    
   %                                                                                                                                                                                                                                                                                                                                	   	       	   
       v   
    ^˺   *          5      dThe operational, rule-based, problems is a very interesting use-case for multi-objective approaches.5_                    	                                                                                                                                                                                                                                                                                                                                   	          	          v       ^   +       
   5      IThe work presents results on an indirect algorithm selection performances5_                                                                                                                                                                                                                                                                                                                                                        	          	          v       ^              6                5    5_                       *                                                                                                                                                                                                                                                                                                                                	          	          v       ^              6      *The paper does not makes clearly the point5_                       !                                                                                                                                                                                                                                                                                                                                	          	          v       ^0              7      @The introduction states that the practitionner wants to navigate5_                       ?                                                                                                                                                                                                                                                                                                                                	          	          v       ^3              7      ?The introduction states that the practitioner wants to navigate5_                                                                                                                                                                                                                                                                                                                                                                             v       ^g              8      gThe introduction states that the practitioner wants to manipulate different rulesets, which seems fair.5_                                                                                                                                                                                                                                                                                                                                                                            v       ^n              8      _Section2 states that the practitioner wants to manipulate different rulesets, which seems fair.5_                       	                                                                                                                                                                                                                                                                                                                                                     v       ^̇              8      	However, 5_                                                                                                                                                                                                                                                                                                                                                                            v       ^              9      nHowever, section 3.1 states that the algorithm configuration is a multi-objective method (title of section 3),5_                       V                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^              9      lHowever, section 3 states that the algorithm configuration is a multi-objective method (title of section 3),5_                                                                                                                                                                                                                                                                                                                                                          V          j       v   j    ^              9      while 5_                       `                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^              9      `Section 2 states that the practitioner wants to manipulate different rulesets, which seems fair.5_                                                                                                                                                                                                                                                                                                                                                          V          j       v   j    ^              :      FThe paper does not makes clearly the point of producing Pareto fronts.5_                       .                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^              <      .The paper does not articulate very clearly the5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^                     FThe paper does not articulate very clearly the two problems, it's some5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^                      5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^                     3makes clearly the point of producing Pareto fronts.5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^                     `Section 2 states that the practitioner wants to manipulate different rulesets, which seems fair.5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^               8      WHowever, section 3 states that the algorithm configuration is a multi-objective method,5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^!              8      P, section 3 states that the algorithm configuration is a multi-objective method,5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^!              8      O section 3 states that the algorithm configuration is a multi-objective method,5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^#              8      Nsection 3 states that the algorithm configuration is a multi-objective method,5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^$              8                    8       5_                       	                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^.              7      while the end of section 3.15_                       !                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^2              7      !while the very end of section 3.15_                                                                                                                                                                                                                                                                                                                                                          V          j       v   j    ^s              9      "computed on an archive f any sort.5_                       X                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^u              9      qI would say that's rather a mono-objective problem, if a single objective is optimized, even if this objective is5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^v   ,          :      even if this objective is   #computed on an archive of any sort.5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^͈              9    5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^͉              :       5_                                                                                                                                                                                                                                                                                                                                                          V          j       v   j    ^͕              ;      0The paper states clearly that the classification5_                       *                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^͚              ;      <The paper states clearly how and why that the classification5_                       7                                                                                                                                                                                                                                                                                                                                   V          j       v   j    ^͛              ;      7The paper states clearly how and why the classification5_                                                                                                                                                                                                                                                                                                                                                           V          j       v   j    ^͠   -          ;      	However,    NSection 3 states that the algorithm configuration is a multi-objective method,         ;       5_                                                                                                                                                                                                                                                                                                                                                           _          s       v   j    ^              :       5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^	              :      )Additionally, the begining of the section5_                       C                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^)              :      aAdditionally, it's not clear how the problem has three objectives, while section 3.1 later states5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^*              ;      while section 3.1 later states5_                       
                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^A              =      
I assume that5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^L              =      I assume that the seletion5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^M              =      I assume that the selection5_                       D                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^\              =      ZI assume that the selection is a mono-objective problem in essence, but solved by the mean5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^]   .          >      but solved by the mean5_                                                                                                                                                                                                                                                                                                                                                           _          s       v   j    ^Φ              @      collasped at the very end.5_                                                                                                                                                                                                                                                                                                                                                          _          s       v   j    ^έ   /          @      collapsed at the very end.5_                    "                                                                                                                                                                                                                                                                                                                                       _          s       v   j    ^        "   $   A          "   $   @    5_                    #   
                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^        "   $   A      - figures appear 5_                    #                                                                                                                                                                                                                                                                                                                                      _          s       v   j    ^        "   %   A      - figures and tables appear 5_                    $                                                                                                                                                                                                                                                                                                                                      _          s       v   j    ^        "   $   B      w- figures and tables appear a bit far from their reference in the text, if possible, displaying them earlier would ease    the    #   %   B        the 5_                    #   |                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^        "   %   A      |- figures and tables appear a bit far from their reference in the text, if possible, displaying them earlier would ease the 5_                    #   
                                                                                                                                                                                                                                                                                                                                   _          s       v   j    ^   0    "   $   B      w- figures and tables appear a bit far from their reference in the text, if possible, displaying them earlier would ease     the read.5_                    #   H                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^   1    "   $   A      - figures and tables appear a bit far from their reference in the text, if possible, displaying them earlier would ease the read.5_                       '                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^2              A      DSection 3 would benefits from having this point more clearly stated.5_                       )                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^3              A      DSection 3 would benefits from having thos point more clearly stated.5_                       0                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^4              A      ESection 3 would benefits from having those point more clearly stated.5_                       E                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^5   2          A      FSection 3 would benefits from having those points more clearly stated.5_                       5                                                                                                                                                                                                                                                                                                                                #   H       #   T       v   T    ^F              A    5_                       S                                                                                                                                                                                                                                                                                                                                $   H       $   T       v   T    ^G   3          B      SThe paper states clearly how and why the classification problem is multi-objective.5_                       H                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^S           !   D             !   C    5_                                                                                                                                                                                                                                                                                                                                                         &   H       &   T       v   T    ^ϒ                      I also suggest to justify why5_                       C                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^ϛ              C      DI assume that the selection is a mono-objective problem in essence, 5_                       H                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^ϝ   4           C      ISection 3 would benefits from having those points more clearly explained.5_                       3                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^ϭ              C      Pbut solved by the mean of a multi-objective search, which output Pareto front is5_                                                                                                                                                                                                                                                                                                                                                        &   H       &   T       v   T    ^Ϯ   5          D      which output Pareto front is   4collapsed at the very end with an aggregated metric.5_                       V                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^Ͻ               C      WSection 3 would benefits from having those points more clearly explained and justified.5_                       X                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^Ͽ           !   C      ZSection 3 would benefits from having those points more clearly explained and justified ().5_                                                                                                                                                                                                                                                                                                                                                       &   H       &   T       v   T    ^               D      vSection 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective   search).5_                       }                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^           !   C      Section 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective search).5_                                                                                                                                                                                                                                                                                                                                                       &   H       &   T       v   T    ^   6           D      vSection 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective   	search?).5_                       }                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^           !   C      Section 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective search?).5_                                                                                                                                                                                                                                                                                                                                                       &   H       &   T       v   T    ^               D      vSection 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective   configuration, for instance?).5_                       V                                                                                                                                                                                                                                                                                                                                %   H       %   T       v   T    ^   7       !   C      Section 3 would benefits from having those points more clearly explained and justified (why not using a mono-objective configuration, for instance?).5_                    !                                                                                                                                                                                                                                                                                                                                    &   H       &   T       v   T    ^        !   $   E          !   #   D    5_                    #   A                                                                                                                                                                                                                                                                                                                                (   H       (   T       v   T    ^1        "   $   F      GI think this is a crucial point in order to interpret performance data.5_                    #   H                                                                                                                                                                                                                                                                                                                                (   H       (   T       v   T    ^2   8    "   %   F      HI think this is a crucial point in order to interpret performances data.5_                    %                                                                                                                                                                                                                                                                                                                                    )   H       )   T       v   T    ^}   9    %   '   G    5_                    %                                                                                                                                                                                                                                                                                                                                    *   H       *   T       v   T    ^Ќ        %   (   I          %   '   H    5_                    '   !                                                                                                                                                                                                                                                                                                                                ,   H       ,   T       v   T    ^а        &   (   J      !I assume it's objective functions5_                    '                                                                                                                                                                                                                                                                                                                                   ,   H       ,   T       v   T    ^м        &   (   J      KI assume it's objective functions evaluations, but it should be made clear.5_                    '   P                                                                                                                                                                                                                                                                                                                                ,   H       ,   T       v   T    ^   :    &   )   J      PI will assume it's objective functions evaluations, but it should be made clear.5_                    (                                                                                                                                                                                                                                                                                                                                    -   H       -   T       v   T    ^        '   )   K       5_                    (                                                                                                                                                                                                                                                                                                                                   -   H       -   T       v   T    ^        '   )   K      If it's not,have 5_                    (                                                                                                                                                                                                                                                                                                                                   @          @          v       ^   ;    '   )   K      If it's not, have    (   )   K    5_                    (                                                                                                                                                                                                                                                                                                                                   @          @          v       ^   <    (   +   L          (   *   K    5_                    .                                                                                                                                                                                                                                                                                                                                    B          B          v       ^        -   /   M       .   /   M    5_                    .                                                                                                                                                                                                                                                                                                                                    C          C          v       ^        -   /   N    5_                    /                                                                                                                                                                                                                                                                                                                                    D          D          v       ^        .   0   O      KParameters of the Friedman and Wilcoxon test should be indicated somewhere.5_                    /   O                                                                                                                                                                                                                                                                                                                                D          D          v       ^        .   0   O      OThe parameters of the Friedman and Wilcoxon test should be indicated somewhere.5_                    /   ]                                                                                                                                                                                                                                                                                                                                D          D          v       ^;        .   0   O      hThe parameters of the Friedman and Wilcoxon test should be indicated somewhere around section / table 5.5_                    .                                                                                                                                                                                                                                                                                                                                    D          D          v       ^>        -   1   O       5_                    /                                                                                                                                                                                                                                                                                                                                    F          F          v       ^>        .   0   Q       5_                    /                                                                                                                                                                                                                                                                                                                                    F          F          v       ^\        .   /          ;Section 3 only have one subsection, which is a bit useless.5_                    1                                                                                                                                                                                                                                                                                                                                    E          E          v       ^^        1   3   P       1   2   P    5_                    2                                                                                                                                                                                                                                                                                                                                    F          F          v       ^_        1   3   Q      ;Section 3 only have one subsection, which is a bit useless.5_                    /                                                                                                                                                                                                                                                                                                                                    F          F          v       ^`   =    -   /   Q              .   0   Q       5_                    0                                                                                                                                                                                                                                                                                                                                    I           K           v   _    ^u        /   4   P          0   1   P    5_                    0                                                                                                                                                                                                                                                                                                                                   L           N           v   _    ^y        /   1   S      wTable 5 does not show "the best configurations […] in bold", given that there are some lines where all configurations5_                    0   !                                                                                                                                                                                                                                                                                                                                L           N           v   _    ^}        /   1   S      Table 5, in particular, does not show "the best configurations […] in bold", given that there are some lines where all configurations5_                    0   W                                                                                                                                                                                                                                                                                                                                L           N           v   _    ^с        /   2   S      Table 5, in particular, does not actually show "the best configurations […] in bold", given that there are some lines where all configurations5_                    1                                                                                                                                                                                                                                                                                                                                   M           O           v   _    ^х        0   2   T      9 given that there are some lines where all configurations5_                    1                                                                                                                                                                                                                                                                                                                                   M           O           v   _    ^ё        0   2   T      3There is indeed some lines where all configurations5_                    1   4                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^ѕ        0   2   T      4There are indeed some lines where all configurations   ^are in bold. Please take care of describing and interpreting statistical tests very carefully.   0   2   T      4There are indeed some lines where all configurations5_                    1   B                                                                                                                                                                                                                                                                                                                                L           N           v   _    ^ї        0   3   S      There are indeed some lines where all configurations are in bold. Please take care of describing and interpreting statistical tests very carefully.5_                    1   @                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^ћ        0   2   T      AThere are indeed some lines where all configurations are in bold.5_                    1   e                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^Ѷ        0   2   T      eThere are indeed some lines where all configurations are in bold, even if performances are different.5_                    2   P                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^        1   3   T      QPlease take care of describing and interpreting statistical tests very carefully.5_                    2   R                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^        1   3   T      dPlease take care of describing and interpreting statistical tests very carefully, given the current.5_                    2   Q                                                                                                                                                                                                                                                                                                                                M           O           v   _    ^        1   4   T      cPlease take care of describing and interpreting statistical tests very carefully,given the current.5_                    3                                                                                                                                                                                                                                                                                                                                   N           P           v   _    ^        2   4   U      given the current.5_                    3                                                                                                                                                                                                                                                                                                                                   N           P           v   _    ^        2   4   U      &given the current trend with p-values.5_                    3   #                                                                                                                                                                                                                                                                                                                                N           P           v   _    ^   >    2   4   U      $given the current trend on p-values.5_                    +                                                                                                                                                                                                                                                                                                                                    +           -          v       ^        *   ,   U          Minor revisions   ---------------    5_                    1                                                                                                                                                                                                                                                                                                                                    +           +          v       ^        0   5   R          1   2   R    5_                    *                                                                                                                                                                                                                                                                                                                                    +           +          v       ^   ?    )   *           5_                    /   J                                                                                                                                                                                                                                                                                                                                *           *          v       ^        /   2   U          /   1   T    5_                    0                                                                                                                                                                                                                                                                                                                                   *           *          v       ^$        /   1   V      I assumed that bold means that 5_                   0                                                                                                                                                                                                                                                                                                                                    *           *          v       ^1        /   0          FI assumed that bold means that there is a statistical difference with 5_                  0                                                                                                                                                                                                                                                                                                                                    *           *          v       ^3        /   1   U       5_                 0                                                                                                                                                                                                                                                                                                                                    *           *          v       ^@        /   1   U       5_                 0                                                                                                                                                                                                                                                                                                                                    *           *          v       ^K        /   1   U      ;I'm not sure how to interpret results presented on Talbe 5.5_                 0   G                                                                                                                                                                                                                                                                                                                                *           *          v       ^P        /   1   U      LFor this reason, I'm not sure how to interpret results presented on Talbe 5.5_                 0   G                                                                                                                                                                                                                                                                                                                                *           *          v       ^Q   @    /   1   U      KFor this reason, I'm not sure how to interpret results presented on Tabe 5.5_                 0   G                                                                                                                                                                                                                                                                                                                                *           *          v       ^U        0   2   U    5_                 1                                                                                                                                                                                                                                                                                                                                    Q           V           v   =    ^ҏ        0   7   V          1   2   V    5_    	             3   M                                                                                                                                                                                                                                                                                                                                V           [           v   =    ^қ        3   6   \          3   5   [    5_    
          	   4                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ұ        3   4   ^      Ktable 5 alone is not sufficient to get a clear picture, I suggest to add a    5   7   ]    5_  	            
   5                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ұ        4   5   ^      Ktable 5 alone is not sufficient to get a clear picture, I suggest to add a    6   8   ]    5_  
               6                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ұ        5   6   ^      Ktable 5 alone is not sufficient to get a clear picture, I suggest to add a    7   9   ]    5_    
             7                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ұ        6   7   ^      Ktable 5 alone is not sufficient to get a clear picture, I suggest to add a    8   :   ]    5_              
   4                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ҳ        2   4   ]      Nit is a lot less clear that there is an improvement on the operational result.       3   5   ]       5_  
               4   +                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^Ҵ        3   6   \      +This should be reflected in the conclusion.5_                 6                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ҵ        5   6   ^      :A concise view on the results shown on table 5 would help.   7   9   ]    5_                 7                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^ҵ        6   7   ^      :A concise view on the results shown on table 5 would help.   8   :   ]    5_                 7                                                                                                                                                                                                                                                                                                                                   X           ]           v   =    ^Ҷ        5   7   ]          Jable 5 alone is not sufficient to get a clear picture, I suggest to add a    6   8   ]      Ktable 5 alone is not sufficient to get a clear picture, I suggest to add a 5_                 7                                                                                                                                                                                                                                                                                                                                   W           \           v   =    ^һ        6   8   \      :A concise view on the results shown on table 5 would help.5_                 6   K                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^ҽ        5   7   \      KTable 5 alone is not sufficient to get a clear picture, I suggest to add a 5_                 7                                                                                                                                                                                                                                                                                                                                    W           \           v   =    ^Ҿ        6   8   \      8concise view on the results shown on table 5 would help.5_                 7   :                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^   A    6   8   \      :a concise view on the results shown on table 5 would help.5_                 7   O                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^   B    7   9   \    5_                 )                                                                                                                                                                                                                                                                                                                                    X           ]           v   =    ^   C    (   )           5_                 6   H                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^$        5   7   \      Pa concise view on the results to support the claims made in the results section.5_                 6   _                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^(        5   7   \      `a concise view on the results to support the claims made in the results  and conclusion section.5_                 6   H                                                                                                                                                                                                                                                                                                                                W           \           v   =    ^)   D    5   7   \      aa concise view on the results to support the claims made in the results  and conclusion sections.5_                                                                                                                                                                                                                                                                                                                                                     W           \           v   =    ^<              ]                \    5_                                                                                                                                                                                                                                                                                                                                                     Z           _           v   =    ^i                     QThe paper suffers from a lack of clarity on the articulation of the two problems.5_                                                                                                                                                                                                                                                                                                                                                    Y           ^           v   =    ^k              ^             ^    5_                                                                                                                                                                                                                                                                                                                                                     Z           _           v   =    ^l              _      1)    QThe paper suffers from a lack of clarity on the articulation of the two problems.5_                     T                                                                                                                                                                                                                                                                                                                                Y           ^           v   =    ^n              ^      T1) The paper suffers from a lack of clarity on the articulation of the two problems.5_    !                                                                                                                                                                                                                                                                                                                                                  Z           _           v   =    ^z              _      ature of the5_     "          !      
                                                                                                                                                                                                                                                                                                                                Z           _           v   =    ^{              _      
nature of the5_  !  #          "      5                                                                                                                                                                                                                                                                                                                                Z           _           v   =    ^Ӆ              _      w1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective   6nature of the configuration problem is not convincing.5_  "  $          #                                                                                                                                                                                                                                                                                                                                      Y           ^           v   =    ^Ӊ              ^      1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective nature of the configuration problem is not convincing.5_  #  %          $                                                                                                                                                                                                                                                                                                                                      [           `           v   =    ^Ӧ              `      w1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective   wnature of the configuration problem is not convincing. However, this does not discards the interest of the studyand the5_  $  &          %      w                                                                                                                                                                                                                                                                                                                                Z           _           v   =    ^ӧ              _      1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective nature of the configuration problem is not convincing. However, this does not discards the interest of the studyand the   results.5_  %  '          &                                                                                                                                                                                                                                                                                                                                      Y           ^           v   =    ^ө              ^      1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective nature of the configuration problem is not convincing. However, this does not discards the interest of the studyand the results.5_  &  (          '                                                                                                                                                                                                                                                                                                                                      Y           ^           v   =    ^Ӫ              ^      1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective nature of the configuration problem is not convincing. However, this does not discards the interest of the study and the results.5_  '  )          (      r                                                                                                                                                                                                                                                                                                                                Z           _           v   =    ^Ӭ              _      w1) The paper suffers from a lack of clarity on the articulation of the two problems. In particular, the multi-objective   nature of the configuration problem is not convincing. However, this does not discards the interest of the study or the results.5_  (  *          )                                                                                                                                                                                                                                                                                                                                       Y           ^           v   =    ^Ӯ   E          _                ^    5_  )  +          *   $                                                                                                                                                                                                                                                                                                                                    $           +          v       ^        #   %   _   	       dSection 4.2 does not indicates whether the allocated running time allowed for each runs to converge.   HI think this is a crucial point in order to interpret performances data.   SThe "convergence" metric used in the preliminary experiments may also be indicated.       @Additionally, the unit of this running time should be indicated.   PI will assume it's objective functions evaluations, but it should be made clear.   If it's not, have the preliminary experiments be conducted in the same setting than the experiments (i.e. full load on 24 cores)?    5_  *  ,          +   6                                                                                                                                                                                                                                                                                                                                    $           $          v       ^   F    5   ?   W          6   7   W    5_  +  -          ,                                                                                                                                                                                                                                                                                                                                      $           $          v       ^              _      2) 5_  ,  .          -      '                                                                                                                                                                                                                                                                                                                                %           %          v       ^   G          `      q2) The terminal results on the classification task are not easy to interpret. The claim that there is an improved   (performance is not clearly demonstrated.5_  -  /          .                                                                                                                                                                                                                                                                                                                                       $           $          v       ^   H          _    5_  .  0          /   /                                                                                                                                                                                                                                                                                                                                   %           %          v       ^(        .   0   `      +This should be reflected in the conclusion.5_  /  1          0   /   1                                                                                                                                                                                                                                                                                                                                %           %          v       ^*   I    .   0   `      2This should either be reflected in the conclusion.5_  0  2          1   1                                                                                                                                                                                                                                                                                                                                   1          1          v       ^:   J    0   2   `      HTable 5 alone is not sufficient to get a clear picture, I suggest to add5_  1  3          2   K                                                                                                                                                                                                                                                                                                                                    K           `           v   I    ^Ԅ        J                  UFirst, stated that practitonners need to navigate within pareto-front of classifiers.   0Then, why does MOCA-I returns a single ruleset??   ;Why not using mono-objective with this harmonic mean, then?       UIt's unclear if the configuration is a 3 objective pb, as stated in the introduction,   bor a bi-objective problem, as suggested in section 3 (ruleset size being used to direct searches).       KSec. 4.2 does the set running time allowed for convergence of all the runs?   QWhat is the unit of running time \delta? Is it function calls or wall clock time?   If wallclock times, have the preliminary experiments be conducted in the same setting than the experiments (i.e. full load on 24 cores)?       KParameters of the Friedman and Wilcoxon test should be indicated somewhere.       wTable 5 does not show "the best configurations […] in bold", given that there are some lines where all configurations   ^are in bold. Please take care of describing and interpreting statistical tests very carefully.       fWhile it is sufficiently clear that AC allow for improvement in terms of Pareto front quality metrics,   Nit is a lot less clear that there is an improvement on the operational result.   +This should be reflected in the conclusion.   :A concise view on the results shown on table 5 would help.    5_  2  4          3   K                                                                                                                                                                                                                                                                                                                                    K           K           v   I    ^ԅ   K    J   K           5_  3  5          4                                                                                                                                                                                                                                                                                                                                                   
       v   
    ^ԑ   L          J      #I see three major revisions needed:5_  4  6          5   3                                                                                                                                                                                                                                                                                                                                                 
       v   
    ^Ԫ   M    3   6   K          3   5   J    5_  5  7          6   5   4                                                                                                                                                                                                                                                                                                                                             
       v   
    ^   N    4   7   L      5in terms of the four terminal classification metrics.5_  6  8          7   6   `                                                                                                                                                                                                                                                                                                                                             
       v   
    ^   O    6   8   M    5_  7  9          8   5   4                                                                                                                                                                                                                                                                                                                                             
       v   
    ^   P    4   6   N      5in terms of the four terminal classification metrics,5_  8  :          9   6   $                                                                                                                                                                                                                                                                                                                                6   $       6   &       v   &    ^        5   7   N      agiven that the performance profiles are somehow different (except for F and MCC, as pointed out).5_  9  ;          :   6   :                                                                                                                                                                                                                                                                                                                                6   $       6   &       v   &    ^   Q    5   7   N      bgiven that the performance profiles seem somehow different (except for F and MCC, as pointed out).5_  :  <          ;   6   1                                                                                                                                                                                                                                                                                                                                6   $       6   &       v   &    ^   R    5   7   N      cgiven that the performance profiles seem somehow differents (except for F and MCC, as pointed out).5_  ;              <   D                                                                                                                                                                                                                                                                                                                                    6   $       6   &       v   &    ^,   S    D   F   O          D   F   N    5